<?php
/**
 * @package		简易CMS
 * @subpackage	新闻采集
 * @version		$Id: class_import.php 25 2012-08-29 17:55:19Z htmambo@gmail.com $
 * @author		Hoping
 * @copyright	Copyright (C) 2009 - 2010 Hoping Software Studio. All rights reserved.
 */

 if (!defined('IN_JYCMS')) {
	exit('Access Denied');
}

class import {

	var $url = '';
	var $charset = 'auto'; //编码
	var $rule; //正文获取规则
	var $replace; //正文的替换规则
	var $content = array(); //获取到的正文内容，含多页
	var $urls = array(); //已经读取的多页链接
	var $finshed = false; //成功采集
	var $sourcehtml = '';
	var $snoopy;
	var $tmpfilelist = array(); //保存采集时的临时文件

	function __construct() {
		$this->import();
	}

	function import() {
		global $snoopy;
		@ini_set('memory_limit', '256m');
		if (!$snoopy) {
			require_once libfile('class/snoopy');
			$snoopy = new Snoopy();
		}
		$this->snoopy = & $snoopy;
		$this->snoopy->rawheaders["Pragma"] = "no-cache";
		$this->snoopy->maxredirs = 2;
		$this->snoopy->offsiteok = false;
		$this->snoopy->expandlinks = false;
		if (version_compare(PHP_VERSION, '5') == -1) {
			register_shutdown_function(array(&$this, '__destruct'));
		}
	}

	public function __destruct() {
		//所有工作都完成了，可以清除采集时产生的临时文件了
		if ($this->finshed && count($this->tmpfilelist)) {
			//echo '共有临时文件'.count($this->tmpfilelist).'个<br />';
			foreach ($this->tmpfilelist as $v) {
				@unlink($v);
			}
			$this->tmpfilelist = array();
		}
	}

	/**
	 * 获取链接的正文
	 *
	 * @param 数组($url,$rule,$replace,$charset) $row
	 */
	function geturl($row = array()) {
		$row['url'] = trim($row['url']);
		if (!$row || !$row['url']) {
			//未定义参数或者未定义URL
			return false;
		}
		$this->url = $row['url'];
		//测试
		//$this->url = 'http://it.dbw.cn/system/2010/03/21/052411596.shtml';
		//如果没有指定采集规则，则去数据库里找一下
		if (!$row['rule'] && !$row['replace']) {
			//加载采集规则
			if (!$this->getrule()) {
				$this->charset = 'auto';
				$this->rule = '';
				$this->replace = '';
			}
		} else {
			$this->processrule($row['rule'], $row['replace']);
			$this->charset = $row['charset'];
		}
		$this->getcontent($this->url);
		ksort($this->urls);
		ksort($this->content);
		if ($this->content) {
			$i = 0;
			$contents = $this->content;
			$this->content = array();
			$tmp = '';
			foreach ($contents as $v) {
				$tmp.=trim($v);
				if (strlen($tmp) > 500) {
					$i++;
					$this->content[] = $tmp;
					$tmp = '';
				}
			}
			if ($tmp) {
				$this->content[] = $tmp;
				$i++;
			}
		} else {
			return false;
			//echo 'not readed!';
		}
		//pre($this->content, 1);
		$message = implode("\r\n<p>###NextPage###</p>\r\n", $this->content);
		//$message = str_replace('<p></p>', '', $message);
		preg_match_all("/<p>(.+?)<\/p>/iU", $message, $matches);
		if (count($matches[1])) {
			$rows = $matches[1];
			$this->content = $message = $testmsg = array();
			//pre($rows);
			$i = 0;
			foreach ($rows as $v) {
				//echo $v.'<br>';
				//echo code($v);
				$v = trim($v);
				$k = md5($v);
				//智能读取后，整段内容重复且长度较小的话，直接删除掉
				//所以，如果目标内容是多页的话，读取的结果会更加准确。当然，已经忽略掉的内容就没办法了
				if ($v == '###NextPage###') {
					$i++;
					$message[] = $v;
					//$this->content[] = '<p>' . implode('</p><p>', $message) . '</p>';
					//$message = array();
				} elseif (isset($testmsg[$k]) && strlen($v) < 180) {
					unset($message[$k]);
				} else {
					$message[$k] = $v;
					$testmsg[$k] = $v;
				}
			}
			$this->content[] = '<p>' . implode('</p><p>', $message) . '</p>';
		}
		//echo count($this->content);
		$result = implode("###NextPage###", $this->content);
		//exit(code($result));
		$result = str_replace('#`#', "、", $result);
		$result = str_replace('[.', "【", $result);
		$result = str_replace('.]', "】", $result);
		$result = str_replace('?\\0', "&nbsp;", $result);
		//$result = str_replace('"', "'", $result);
		$result = daddslashes($result);
		return $result;
	}

	/**
	 * 获取本次采集的url的规则
	 *
	 * @return 是否存在规则
	 */
	function getrule() {
		global $_G;
		$url = $this->url;
		//return false;
		loadcache('contentrule');
		$contentrule = $_G['cache']['contentrule'];
		if ($contentrule) {
			//echo '搜索与'.$url.'匹配的规则<br>';
			$find = array();
			foreach ((array) $contentrule as $k => $v) {
				if ($url != str_ireplace($v['url'], '', $url)) {
					//获取最长匹配的
					if (strlen($v['url']) > strlen($find['url'])) {
						$find = $v;
					}
				}
			}
			if (count($find) > 0) {
				$this->processrule($find['rule'], $find['replace']);
				$this->charset = $find['charset'];
				return $find;
			}
			else
				return false;
		}
		return false;
	}

	/**
	 * 采集指定网址的内容
	 *
	 * @param mixed $url
	 */
	function getcontent($url) {
		$dirname = dirname($url);
		$filename = basename($url);
		$ext = "." . trim(substr(strrchr($filename, "."), 1));
		$basename = str_ireplace($ext, '', $filename);
		if (!$this->urls[$basename]) { //还没有采集这个链接
			$this->urls[$basename] = $filename;
			//获取第一页
			//国外免费空间限制太多，所以，在采集多页内容时得保存一下临时结果
			//等到入库以后，再把这些临时文件删除掉
			$fn = cmsFS::formatpath(ROOT_PATH . './data/tmp/' . md5($url), false);
			$this->tmpfilelist[] = $fn;
			//@unlink($fn);
			if (file_exists($fn)) {
				$html = '';
				cmsFS::read($fn, $html);
			} else {
				if ($this->snoopy->fetch($url)) {
					$html = $this->snoopy->results;
				} else {
					$html = dfsockopen($url);
				}
				if ($html) {
					cmsFS::write($fn, $html);
				} else {
					//trigger_error("读取 $url 时出错！");
					return false;
				}
			}
			if (!$html) {
				return false;
			}
			$this->snoopy->results = $html;
			//pre($this->snoopy->htmlheader);exit;
			$charset = trim($this->charset);
			if (!$charset || $charset == 'auto') {
				$charset = $this->snoopy->htmlheader['charset'];
			}
			if (!$charset) {
				preg_match("~charset=([^>]+)>~isU", $html, $matches);
				$charset = $matches[1];
				$charset = str_replace(array('"', "'", '/', ' ', '>'), '', $charset);
				if (!$charset)
					$charset = "gb2312";
			}
			$this->charset = $charset;
			//转换编码
			$html = diconv($html, $charset, 'utf-8');
			if (!$this->sourcehtml) {
				$this->sourcehtml = $html; // = $this->formatcontent($html, $url, 0);
				$this->basename = $basename;
			}
			//exit($html);
			if (!$this->snoopy->results) {
				$this->snoopy->results = $html;
			}
			//自动采集多页
			//如果不是论坛采集，则去判断多页，因为论坛的话，通常只采集第一页或者只采集主题贴，故不需要处理多页
			if (!strpos($this->url, "bbs")) {
				//获取相关的链接列表
				$urls = $this->snoopy->getrelationlink($url);
				if (count($urls) > 0) {
					foreach ($urls as $k => $v) {
						if (!$this->urls[$k]) {
							$this->getcontent($v);
						}
					}
				}
			}
			//以下为处理一些特殊字符所导致的入库问题
			$html = str_replace(chr(255), '', $html);
			$html = str_replace(chr(32) . chr(169) . chr(32), '&nbsp;(C)&nbsp;', $html);
			$html = str_replace(chr(32) . chr(169) . '1', '&nbsp;(C)&nbsp;1', $html);
			$html = str_replace(chr(32) . chr(169) . '2', '&nbsp;(C)&nbsp;2', $html);
			$html = str_replace('&copy;', '(C)', $html);
			$content = '';
			//下来开始处理正文过滤以及截取规则
			if ($this->rule) {
				//处理采集规则
				preg_match("~" . $this->rule . "~isU", $html, $matches);
				unset($matches[0]);
				if ($matches[1]) {
					foreach ($this->rules as $k => $v) {
						if ($v == 'content') {
							//正文内容处理部分可以有多个，这样可以采集论坛主题及贴子
							$result = trim($matches[$k]);
							$result = $this->formatcontent($result, $url, 0);
							$result = $this->processrepl($result);
							$content .= $result;
						}
					}
				}
				//采集规则处理结束
			} elseif ($this->replace) {
				//没有规则，直接格式化替换
				$html = $this->formatcontent($html, $url);
				$content = $this->processrepl($html);
			} else {
				//啥规则都没写，系统智能读取
				$content = $this->formatcontent($html, $url);
				if (!$content)
					$content = $html;
			}
			if ($content) {
				$content = censor($content, 1);
				$this->content[$basename] = $content;
			} else {
				$this->content[$basename] = $html;
			}
		}
	}

	function processrepl($html) {
		//检测替换规则
		if ($this->replace) {
			foreach ($this->replace as $v) {
				$v['find'] = str_replace('\\\\', '\\', $v['find']);
				$html = preg_replace("@" . $v['find'] . "@isU", $v['replace'], $html);
			}
		}
		return $html;
	}

	function processrule($rule = '', $replace = '') {
		$this->rule = '';
		$this->replace = array();
		if ($rule) {
			$rule = str_replace("/", '\/', $rule);
			$rule = str_replace("#", '\#', $rule);
			$rule = str_replace("(", '\(', $rule);
			$rule = str_replace(")", '\)', $rule);
			$rule = str_replace("?", '\?', $rule);
			preg_match_all('/\{([a-zA-Z]+)\}/i', $rule, $matches);
			if ($matches[1]) {
				$i = 1;
				foreach ($matches[1] as $k => $v) {
					$v = strtolower($v);
					if ($v !== 'ignore') {
						$this->rules[$i] = $v;
						if ($v !== 'content') { //除内容字段外，其它字符均不得包含引号与双引号
							$matches[1][$k] = "([^'][^>]" . '[^"]' . "+)";
						} else {
							$matches[1][$k] = "(.*?)";
						}
						$i++;
					} else {
						$matches[1][$k] = ".*?";
					}
				}
			}
			$this->rule = str_ireplace($matches[0], $matches[1], $rule);
		}
		$replace = trim($replace);
		if ($replace) {
			$replace = explode("<==newline==>", $replace);
			$replaces = array();
			foreach ($replace as $v) {
				$v = explode("<==>", $v);
				if (count($v) == 1) {
					$v[0] = trim($v[0]);
				}
				$v[1] = trim($v[1]);
				if ($v[1] == 'BR') {
					$v[1] == '\n';
				}
				$this->replace[] = array("find" => $v[0], "replace" => $v[1]);
			}
		}
	}

	function formatcontent($sourcehtml, $referurl, $cut = true) {
		//替换掉一些无用的格式
		$sourcehtml = str_replace("/>", ">", $sourcehtml);
		$sourcehtml = str_replace(" >", '>', $sourcehtml);
		$a = array('object', 'embed', 'style', 'script', 'form', 'head', 'textarea');
		//$a = array('textarea');
		foreach ($a as $v)
			$sourcehtml = cuthtmltag($sourcehtml, $v);
		//清除display:none的部分
		//<span style="display:none"></span>
		$lists = array("span", "div");
		foreach ($lists as $v) {
			$regex = '/<' . $v . '[\x20-\x7f]+display:none[\x20-\x7f]+>.{0,2000}<\/' . $v . '>/isU';
			$sourcehtml = preg_replace($regex, '', $sourcehtml);
		}
		//echo code($sourcehtml);
		//替换掉<body>....</body>以外的内容
		//$sourcehtml = preg_replace('/.*?<\/head/isU', '</head', $sourcehtml);
		//清除垃圾信息
		$sourcehtml = preg_replace("/\<\!\-\-[^>]+>/ies", '', $sourcehtml);
		$sourcehtml = preg_replace('/<([^\s]+)\s[^>]*>[\s|\t|\n|\r]*<\/\\1>/ies', '', $sourcehtml);
		$searchcursory = array(
			"/([\r\n])\\s+/",
			"/\\<(table|div)[^\\>]*?\\>/si",
			"/\\<\\/(table|div)\\>/si"
		);
		$replacecursory = array(
			"\\1",
			"\n\n###table div explode###\n\n",
			"\n\n###table div explode###\n\n"
		);

		$searchaborative = array(
			"/&.{0,4}lsquo;/si",
			"/&.{0,4}rsquo;/si",
			"/&.{0,4}ldquo;/si",
			"/&.{0,4}rdquo;/si",
			"/\\<(iframe)[^\\>]*?\\>.*?\\<\\/(\\1)\\>/si",
			"/\\<[\\/\\!]*?[^\\<\\>]*?\\>/si",
			"/\t/",
			"/[\r\n]+/",
			"/(^[\r\n]|[\r\n]\$)+/",
			"/&(nbsp|#160|\t);/i",
			"/&(iexcl|#161);/i",
			"/&(cent|#162);/i",
			"/&(pound|#163);/i",
			"/&(copy|#169);/i",
			"/&#(\\d+);/e",
			"/<wbr>/i",
			"/\\<([a-zA-Z0-9]+)[^\\>]*?\\>[\s|\t|\r|\n]+\\<\\/(\\1)\\>/si"
		);
		$replaceaborative = array(
			"‘",
			"’",
			"“",
			"”",
			"",
			"",
			" ",
			"\n",
			"",
			" ",
			chr(161),
			chr(162),
			chr(163),
			chr(169),
			"",
			'',
			''
		);
		$arrayrobotmeg = array();
		//echo code($sourcehtml);
		$sourcetext = preg_replace($searchcursory, $replacecursory, $sourcehtml);
		//exit(code($sourcetext));
		$sourcetext = str_replace('&mdash;', '…', $sourcetext);
		$sourcetext = str_replace('&middot;', '.', $sourcetext);
		$sourcetext = str_replace("&middot;", '.', $sourcetext);
		$sourcetext = str_replace("：", ':', $sourcetext);
		$sourcetext = str_replace("；", ';', $sourcetext);
		$sourcetext = str_replace("、", '#`#', $sourcetext);
		$sourcetext = str_replace("，", ',', $sourcetext);
		$sourcetext = str_replace("？", '?', $sourcetext);
		$sourcetext = str_replace("。", '##.##', $sourcetext);

		$sourcetext = str_replace("【", '[.', $sourcetext);
		$sourcetext = str_replace("】", '.]', $sourcetext);
		//echo code($sourcetext);
		//检查文中的图片
		preg_match_all("/<img([^>]+)>/iU", $sourcetext, $imglist);
		unset($imglist[0]);
		if ($imglist[1]) {
			$sourcetext = preg_replace("/<img([^>]+)>/iU", "<p>###imgbegin###\\1###imgend###</p>", $sourcetext);
			foreach ($imglist[1] as $k => $v) {
				$alt = explode(" ", trim($v));
				$imgart = array();
				foreach ($alt as $i) {
					$i = str_replace(array("'", '"'), '', $i);
					$i = explode("=", trim($i));
					$imgart[$i[0]] = $i[1];
					for ($kk = 2; $kk < count($i); $kk++) {
						if (isset($i[$kk])) {
							$imgart[$i[0]] .= "=" . $i[$kk];
						}
					}
				}
				if (isset($imgart['width']) && $imgart['width'] < 100) {
					$sourcetext = str_replace("###imgbegin###" . $v . "###imgend###", '', $sourcetext);
				} else if (isset($imgart['height']) && $imgart['height'] < 100) {
					$sourcetext = str_replace("###imgbegin###" . $v . "###imgend###", '', $sourcetext);
				} else {
					$src = trim($imgart['src']);
					if (!$src) {
						$sourcetext = str_replace("###imgbegin###" . $v . "###imgend###", '', $sourcetext);
					} else {
						if (substr($src, 0, 7) != 'http://') {
							$src = FillUrl($referurl, $src);
						}
						$sourcetext = str_replace("###imgbegin###" . $v . "###imgend###", "###imgbegin### src=" . $src . "###imgend###", $sourcetext);
					}
				}
			}
		}

		//	pre($imglist);
		$arraysource = explode("\n\n###table div explode###\n\n", $sourcetext);
		//echo code($sourcetext);
		$arraycell = array();
		foreach ($arraysource as $value) {
			$cell = array("code" => $value, "text" => preg_replace("/[\n\r\\s]*?/is", "", preg_replace($searchaborative, $replaceaborative, $value)));
			if ($cell['text'] != "") {
				$arraycell[] = $cell;
			}
		}
		$arraymessage = array();
		$leachmessage = "";
		foreach ($arraycell as $value) {
			$arraymessage[] = $value['code'];
		}
		$leachmessage = preg_replace("/\\<(p|br)[^\\>]*?\\>/si", "\n", implode("\n", $arraymessage));
		$arraymessage = explode("\n", preg_replace($searchaborative, $replaceaborative, $leachmessage));
		$leachmessage = "";
		foreach ($arraymessage as $value) {
			$value = preg_replace('/[\n|\r|\s|\t]*/is', '', $value);
			$value = trim($value);
			if (strlen($value) > 1) {
				$leachmessage .= "<p>" . trim($value) . "</p>\n";
			}
		}
		//echo code($leachmessage);
		$leachmessage = preg_replace("/<p>[^>]+>([^<]+)<\/p>/si", "<p>\\1</p>", $leachmessage);
		$leachmessage = preg_replace("/<p>[^>]+><\/p>/si", "", $leachmessage);
		//echo code($leachmessage);
		$leachmessage = str_replace('&nbsp;', ' ', $leachmessage);

		$leachmessage = str_replace('&nbsp;', ' ', $leachmessage);
		$leachmessage = str_ireplace('&lt;p&gt;', '<p>', $leachmessage);
		$leachmessage = str_ireplace('&lt;/p&gt;', '</p>', $leachmessage);
		$leachmessage = str_ireplace('&quot;', '"', $leachmessage);
		//$leachmessage = preg_replace('@\<\!\-\-.*\-\-\>@ise', '', $leachmessage);
		//清除空行、空段
		$leachmessage = preg_replace('/<p>[\n|\r|\t|\s]*<\/p>/isU', "", $leachmessage);
		if ($cut) {
			//echo code($leachmessage);
			$leachmessage = diconv(str_replace("，", ',', $leachmessage), CHARSET, 'gb2312');
			for ($i = 1; $i < 81; $i++)
				$leachmessage = preg_replace('/<p>([\[\]a-zA-Z0-9\s\-\x7f-\xff\!\>\/\:\|]{' . $i . '})<\/p>/isU', "", $leachmessage);
			for ($i = 1; $i < 81; $i++)
				$leachmessage = preg_replace('/<p>([\[\]&gt;\:\(\)\s\x7f-\xff]{' . $i . '})<\/p>/isU', "", $leachmessage);
			$leachmessage = diconv($leachmessage, 'gb2312', CHARSET);
			$leachmessage = str_replace("\0?", ' ', $leachmessage);
			//exit(code($leachmessage));
		}
		//echo code($leachmessage);
		//恢复图片
		$leachmessage = str_replace('###imgbegin###', '<img ', $leachmessage);
		$leachmessage = str_replace('###imgend###', '>', $leachmessage);
		//清除空行、空段
		$leachmessage = preg_replace('/[\s|\r|\n|\t]*<img[\s|\r|\n|\t]*src=[\s|\r|\n|\t]*>[\s|\r|\n|\t]*/isU', '', $leachmessage);
		$leachmessage = preg_replace('/<p><\/p>[\s]+/isU', "", $leachmessage);
		$leachmessage = preg_replace('/<\/p>[\s]+<p>/isU', '</p><p>', $leachmessage);
		$leachmessage = preg_replace('/<\/p><p>/isU', "</p>\n<p>", $leachmessage);
		$leachmessage = preg_replace("/<p>.{1,4}a[^<]+<\/p>/si", '', $leachmessage);
		//清除版权信息
		$leachmessage = preg_replace("/<p>[\s|\r|\n|\t]*copyright[^<]+<\/p>/si", '', $leachmessage);
		$leachmessage = preg_replace("/<p>·[^<]+<\/p>/si", '', $leachmessage);
		$leachmessage = preg_replace("/<p>\[\.[^\.]+\.\]<\/p>/si", '', $leachmessage);
		$leachmessage = str_replace('##.##', '。', $leachmessage);
		$leachmessage = str_ireplace('<p>|</p>', '', $leachmessage);
		$leachmessage = trim($leachmessage);
		$leachmessage = str_replace('#`#', "、", $leachmessage);
		$leachmessage = str_replace('[.', "【", $leachmessage);
		$leachmessage = str_replace('.]', "】", $leachmessage);

		return $leachmessage;
	}

	function geturlcontent($url, $charset) {
		if ($this->snoopy->fetch($url)) {
			$text = $this->snoopy->results;
			if (!$charset || $charset == 'auto') {
				$charset = $snoopy->htmlheader['charset'];
			}
			if (!$charset)
				$charset = "gb2312";
			$text = diconv($text, $charset);
			return $text;
		} else {
			trigger_error("读取 $url 时出错！");
			return false;
		}
	}

	function addslash($rule = '') {
		$rule = str_replace("/", '\/', $rule);
		$rule = str_replace("#", '\#', $rule);
		$rule = str_replace("(", '\(', $rule);
		$rule = str_replace(")", '\)', $rule);
		$rule = str_replace("?", '\?', $rule);
		return $rule;
	}

}

